package scrapy;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.HttpClientUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

/**
 * @Author: huangkunming
 * @Date: 2023/12/1 16:07
 * @Description:
 */
public class ShopifyReviewScrapy {

    public static void main(String[] args) {
        String shopifyUrl = "https://goosecreekcandle.com/products/jolly-3-wick-candyland-candle";

        URL url;
        try {
            url = new URL(shopifyUrl);
        } catch (MalformedURLException e) {
            throw new RuntimeException(e);
        }
        String domain = url.getProtocol() + "://" + url.getHost();

        CloseableHttpClient httpClient = HttpClients.createDefault();

        CloseableHttpResponse response = null;
        //2.创建get请求，相当于在浏览器地址栏输入 网址
        HttpGet request = new HttpGet(shopifyUrl);
        request.setHeader("Origin",domain);
        request.setHeader("Referer",domain + "/");
        request.setHeader("User-Agent","");
        try {
            //3.执行get请求，相当于在输入地址栏后敲回车键
            response = httpClient.execute(request);
            //4.判断响应状态为200，进行处理
            if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                //5.获取响应内容
                HttpEntity httpEntity = response.getEntity();
                String html = EntityUtils.toString(httpEntity, "utf-8");

                // 解析HTML
                Document document = Jsoup.parse(html);

                // 查找包含特定属性和值的<script>标签
                Elements scriptElements = document.select("script[data-oke-metafield-data]");

                if (scriptElements.size() < 2) {
                    return;
                }
                String content = scriptElements.get(1).html();
                JSONObject jsonObject = JSONObject.parseObject(content);
                JSONArray reviews = jsonObject.getJSONArray("reviews");
                String reviewsNextUrl = jsonObject.getString("reviewsNextUrl");

                getReviews(reviews);

                while (StringUtils.isNotBlank(reviewsNextUrl)) {
                    HttpGet apiGet = new HttpGet("https://api.okendo.io/v1" + reviewsNextUrl);
                    response = httpClient.execute(apiGet);
                    if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                        HttpEntity apiHttpEntity = response.getEntity();
                        String resp = EntityUtils.toString(apiHttpEntity, "utf-8");

                        JSONObject apiJsonObject = JSONObject.parseObject(resp);
                        JSONArray apiReviews = apiJsonObject.getJSONArray("reviews");
                        getReviews(apiReviews);

                        reviewsNextUrl = apiJsonObject.getString("reviewsNextUrl");
                    } else {
                        return;
                    }
                }

            } else {
                //如果返回状态不是200，比如404（页面不存在）等，根据情况做处理，这里略
            }
        } catch (IOException e) {
        } finally {
            //6.关闭
            HttpClientUtils.closeQuietly(response);
            HttpClientUtils.closeQuietly(httpClient);
        }
    }

    private static void getReviews(JSONArray reviews) {
        Iterator<Object> iterator = reviews.stream().iterator();
        while (iterator.hasNext()) {
            Object next = iterator.next();
            JSONObject review = JSONObject.parseObject(next.toString());
            String reviewId = review.getString("reviewId");
            String dateCreated = review.getString("dateCreated");
            Date date = new Date(dateCreated);
            String body = review.getString("body");
            Integer rating = review.getInteger("rating");
            Integer helpfulCount = review.getInteger("helpfulCount");
            JSONArray media = review.getJSONArray("media");
            if (CollectionUtils.isNotEmpty(media)) {
                Iterator<Object> imageIt = media.stream().iterator();
                while (imageIt.hasNext()) {
                    Object next1 = imageIt.next();
                    JSONObject image = JSONObject.parseObject(next1.toString());
                    if ("image".equals(image.getString("type"))) {
                        String fullSizeUrl = image.getString("fullSizeUrl");
                    }
                }
            }
            JSONObject reviewer = review.getJSONObject("reviewer");
            String displayName = reviewer.getString("displayName");
            System.out.println(displayName + "============" + body);
        }
    }

}
